
import getopt, sys


def main():
    
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hi:o:v")
    except getopt.GetoptError:
        # print help information and exit:
        usage()
        sys.exit(2)
        
    inputFile = ''
    outputFile = ''
    
    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
        if o in ("-i", "--input"):
            inputFile = a            
        if o in ("-o", "--output"):
            outputFile = a            
        if o == "-v":
            usage()
            sys.exit()
        if o == "--version":
            showVersion()
            sys.exit()
    # ...
    
#    print "----inputFile=" + inputFile
#    print "----outputFile=" + outputFile
#    
#    for o in args:
#        print "----args=" + o

    str_pagePath = inputFile
    str_outputPath = outputFile
    
    htmlKey = '<a href="'
    keyLen = len(htmlKey)
    
    str_NextPageKey = "<a href='Next.htm'"
    
    str_tmp = ""
    str_listPage = ''
    
    str_tmp = str_pagePath + '\\'
    if str_tmp.find('\\\\')<0:  #fix b:\path\ or b:\path
        str_pagePath = str_tmp
    
    # default output path = input path
    if len(str_outputPath)==0:
        str_outputPath = str_pagePath
        
    if str_pagePath.find('.htm')<0:        
        str_listPage = str_pagePath + "list.htm"
    else :
        # fix me ,get path
        str_listPage = str_pagePath
#        print str_pagePath
#    else:
#        print "else"
#    print str_pagePath    
#    sys.exit()
    
    all_the_text = readFile(str_listPage)
    
    processedId = readFile(get_idListFileName(str_pagePath))
#    print "processedId=" + processedId
#    print "---------"
    if len(processedId)==0:
        processedId = "##"
    s = all_the_text
    s = s.replace("<a","\n<a")
    prior_id = ''
    
    IDlist = [-1]
    htmlFullBody = ''
    ssplit = s.split('\n')

    for tmpLine in ssplit :
        if len(tmpLine)> keyLen :
            if not cmp(tmpLine[0:keyLen],htmlKey[0:keyLen]) :
                #print tmpLine
                nPos = tmpLine.find('.htm">')
                #print nPos
                str_fileID = tmpLine[keyLen:nPos]
                #print '========== Start process ... ' + str_fileID
                #print (len(prior_id))
                if len(prior_id)>0:
                    if processedId.find('#' +str(prior_id)+ '#')<0:
                        # process
                        str_tmpPageFile = str_pagePath + prior_id + ".htm"
                        #print 'prior page( str_tmpPageFile ) = ' + str_tmpPageFile
                        str_Page = readFile(str_tmpPageFile)
                        if (not str_Page) or (len(str_Page)<1):
                            print "exit #1"
                            sys.exit()
                        if len(str_Page) > 0 :
                            #print "---len=" + str(len(str_Page))
                            if str_Page.find(str_NextPageKey)>0:
                                tp = str_Page
                                tp = tp.replace(str_NextPageKey,"<a href='" + str_fileID + ".htm'")
                                print '[' + prior_id + '.htm' + "].Next Page" + ' ==> ' + str_fileID + '.htm'
                                writeFile(str_outputPath + prior_id + ".htm", tp)
                                IDlist.append(prior_id)
                if len(str_fileID)>0:
                    prior_id = str_fileID
    #log IDlist
    if len(IDlist)>1:
        update_idListFile(IDlist,str_outputPath)


######### ######### ######### ######### ######### usage
def usage():
    print "----in usage"


######### ######### ######### ######### ######### get_idListFileName
def get_idListFileName(path,fileName='processed.txt'):
    #print 'get_idListFileName=' + path + fileName
    return path + fileName


######### ######### ######### ######### ######### update_idListFile
def update_idListFile(IDlist,path):
    #IDlist     =list of processed page id
    #path       =path of processed page
    fileName = ''
    fileBuff = ""
    fileName = get_idListFileName(path)
    fileBuff = readFile(fileName)
    for tmpID in IDlist:
        fileBuff = fileBuff + '#' + str(tmpID) + '#\n'
    writeFile(fileName, fileBuff)
    return 0

######### ######### ######### ######### ######### fileExists
def fileExists(fileName):
    file_object = None
    try:
        file_object = open(fileName)       
    except IOError:                     
#        print "The file does not exist, exiting gracefully"
        return False
    finally:
        if file_object:
            file_object.close( )
#    print "This line will always print" 
    return True

######### ######### ######### ######### ######### readFile
def readFile(fileName):
    all_the_text = None
    
    if not fileExists(fileName):
        return ''
    file_object = open(fileName)  
    try:  
        all_the_text = file_object.read( )
#    except IOError, message:    # file open failed
#        print >> sys.stderr, "File could not be opened:", message
#        print "**" + "File could not be opened:", message
#        return ''
    except IOError:    # file open failed
        print "**" + "File could not be opened"
        return ''
    finally:
        file_object.close( )
    return all_the_text

######### ######### ######### ######### ######### writeFile
def writeFile(fileName, fileBody):
    # open file
    try:
        file = open(fileName, "w" ) # open file in write mode
    except IOError, message:    # file open failed
        print >> sys.stderr, "File could not be opened:", message
        sys.exit( 1 )
    print >> file, fileBody        # write entry to file
    file.close()
    
######### ######### ######### ######### ######### __name__
if __name__ == "__main__":
    main()
    